import asyncio
import json
import time

###########################################
# 同项目调用基础包
import os
import sys
import time

filepath = os.path.abspath(__file__)
pathlist = filepath.split(os.sep)
pathlist = pathlist[:-5]
TopPath = os.sep.join(pathlist)
sys.path.insert(0, TopPath)
print(TopPath)
############################################

from pymongo.errors import DuplicateKeyError

from re_common.baselibrary.utils.basedir import BaseDir
from re_common.baselibrary.utils.basefile import BaseFile
from re_common.baselibrary.utils.basegzip import BaseGzip
from re_common.baselibrary.utils.basemotor import BaseMotor

bs = BaseMotor()
bs.AsyncIOMotorClient(
    "mongodb://192.168.31.30:32417/htmljson.paper_jinjiribao?authSource=htmljson",
    "htmljson")
# bs.get_col("jss_patent_page")
bs.get_col("paper_renminribaohw")
# bs.get_col("test")

##############################################################
# async def insert_file(filename, bs):
#     try:
#         for line in BaseGzip(100).read_gz_file(filename):
#             try:
#                 dicts = json.loads(line)
#                 dicts["_id"] = dicts["cid"] + "_" + dicts["pagenum"]
#                 try:
#                     await bs.insert_one(dicts)
#                 except DuplicateKeyError:
#                     pass
#             except:
#                 pass
#     except Exception as e:
#         print(filename)
#         BaseFile.single_add_file("./errfiles.txt", filename + "\n")
#         print(e)
#
#
# start_time = time.time()
# lists = []
# # for file in BaseDir.get_dir_all_files(r"\\192.168.31.123\f\download\wf_zl\download\pages_big_json_over"):
# #     lists.append(insert_file(file, bs))
# #     # asyncio.get_event_loop().run_until_complete(insert_file(file,bs))
# #
# for file in BaseDir.get_dir_all_files(r"\\192.168.31.66\e\download\wf_zl\download\pages_big_json_mg"):
#     lists.append(insert_file(file,bs))
# #
# print("***********")
# asyncio.get_event_loop().run_until_complete(asyncio.wait(lists))
# print(time.time() - start_time)

##############################################################




##############################################################

# async def insert_file(filename, bs):
#     try:
#         for line in BaseGzip(100).read_gz_file(filename):
#             try:
#                 dicts = json.loads(line)
#                 dicts["_id"] = dicts["rawid"]
#                 dicts['export_stat'] = 0
#                 try:
#                     await bs.insert_one(dicts)
#                 except DuplicateKeyError:
#                     pass
#             except:
#                 pass
#     except Exception as e:
#         print(filename)
#         BaseFile.single_add_file("./errfiles_2.txt", filename + "\n")
#         print(e)
#
#
# start_time = time.time()
# lists = []
#
# for file in BaseDir.get_dir_all_files(r"\\192.168.31.66\e\download\wf_zl\download\detail_big_json_mg"):
#     lists.append(insert_file(file,bs))
#
# print("***********")
# asyncio.get_event_loop().run_until_complete(asyncio.wait(lists))
# print(time.time() - start_time)

##############################################################
# f = open(file="./mongorawid.txt", mode="w", encoding="utf-8")


# async def dic_hook(doc):
#     print(doc["_id"])
#     # BaseFile.single_add_file("./mongorawid.txt", doc["_id"] + "\n")
#     f.write(doc["_id"] + "\n")


# asyncio.get_event_loop().run_until_complete(bs.find(dic_hook, {"export_stat": 1}))
# f.close()

# with open("./sqlid.txt", "r", encoding="utf-8") as f:
#     a = set(f.readlines())
# with open("./mongorawid.txt", "r", encoding="utf-8") as f:
#     b = set(f.readlines())
#
# c =a-b
# d = set()
# for line in c:
#     line = line.strip()
#     d.add(line)
#
# sql = "update `pagess` set stat=0 where cid_pagenum in {}".format(tuple(d))
# print(sql)


async def dic_hook(doc):
    encoding = 'ISO-8859-1'
    apparent_encoding = 'UTF-8-SIG'
    jsondicts = doc['jsondicts']
    for k,v in jsondicts.items():
        if k == 'count':
            continue
        # stat = v['stat']
        # if stat == -1:
        #     bs.update({"_id": doc["_id"]}, {"jsondicts.{}.stat".format(k): 0})
        #     bs.update({"_id": doc["_id"]}, {"stathtml": 0})
        #     print("{}-{}更新".format((doc["_id"]), k))
        # else:
        #     print("{}-{}无错误".format((doc["_id"]),k))
        html = v['html']

        try:
            xx = html.encode('ISO-8859-1').decode('UTF-8-SIG')
            bs.update({"_id":doc["_id"]},{"jsondicts.{}.html".format(k):xx})
            print('{}编码成功'.format(v['title']))
        except Exception as e:
            print(e)
            print('{}正确'.format(v['title']))
    # BaseFile.single_add_file("./mongorawid.txt", doc["_id"] + "\n")



asyncio.get_event_loop().run_until_complete(bs.find(dic_hook, {"stat" : 1},{"jsondicts":1,"_id":1}))
